
(defun export-data (&optional file &key (variables t) (types t) (labels t))
"Args: (&optional file dont-save closing &key (variables t) (types t) (labels t))
FILE is a string naming the file which will contain the to-be exported data. If datasheet is open, editable, and edited, the datasheet is saved so that the dataobject is updated (if the dataobject has family, a new dataobject is created). Then, unless DONT-SAVE is T, the updated or new data-object is written to FILE.lsp as a flat ascii file. When VARIABLES is T the file will have variable names in the first line. Similarly, TYPES and LABELS specify that variable types and observation labels will also be saved. The data can be read in with the IMPORT DATA command.
CLOSING is t when datasheet being closed as well." 
  (cond 
    ((not *user-path*)
     (message-dialog (format nil "Sorry. No User Directory.~%You cannot export data.")))
    (t
     (send *current-data* :export-data file))))

(defmeth mv-data-object-proto :export-data (&optional file dont-save
closing &key (variables t) (types t) (labels t))
"Args: (&optional file dont-save closing &key (variables t) (types t) (labels t))
FILE is a string naming the file which will contain the to-be exported data. If datasheet is open, editable, and edited, the datasheet
is saved so that the dataobject is updated (if the dataobject has family, a
new dataobject is created). Then, unless DONT-SAVE is T, the updated or new
data-object is written to FILE.lsp as a flat ascii file. When VARIABLES is T the file will have variable names in the first line. Similarly, TYPES and LABELS specify that variable types and observation labels will also be saved. The data can be read in with the IMPORT DATA command.
CLOSING is t when datasheet being closed as well." 
  (let* ((closed t)
         (string "Export Data to File:")
         (dsob (send *current-data* :datasheet-object))
         (mattype (equal (string-downcase (send self :datatype)) "matrix"))
         (choices) 
         (varnames)
         (datarow)
         (nobs)
         )
 (if (and variables types labels)
     (setf choices '((0 1)))
     (setf choices (choose-subset-dialog "Exported Data Should Include:"
                                (list "Variable Names and Types"
                                      "Observation (Row) Labels")))
     )
    (when choices
          (setf choices (first choices))
          (setf choices (if (member 0 choices)
                            (if (member 1 choices) '(0 1 2) '(0 1))
                            (if (member 1 choices) '(2) '(nil))))
          (when dsob (send dsob :error-check))
          (when (not dont-save)
                (when (not (set-working-directory *user-dir-name*))
                      (set-working-directory "C:\\windows\\desktop"))
                (when
                 (not file) 
                 (setf file
                       #+macintosh(set-file-dialog string "" t)
                       #+msdos    (set-file-dialog string t 
                                       "Plain Text Files(*.TXT)|*.txt|All Files(*.*)|*.*")
                       #+X11    (if file (file-save-dialog string "*.txt" "." file )
                                    #+X11        (file-save-dialog string "*.txt" "."))
                       ))
                (when file (setf *user-dir-name* (get-working-directory))))
          (when (and (send self :datasheet-open)
                     (send (send current-data :datasheet-object) :editable))
                (setf closed (send (send self :datasheet-object) 
                                   :save-datasheet t closing)))

          (when (and file (not dont-save))
                (when closed
                      (when dsob (send dsob :save-datasheet-arguments))
                      (when (and *datasheet* (not (equal dsob *datasheet*)))
                            (send *datasheet* :save-datasheet-arguments))
                      (setf varnames (send *current-data* :active-variables '(all)))
                      (setf typenames (send *current-data* :active-types '(all)))
                      (setf datarow 
                            (if mattype
                                (send self :get-active-data-rows)
                                (row-list (send *current-data* :active-data-matrix '(all)))))
                      (setf nobs (length datarow))
                      (setf file (string-downcase-if-not-X11 file))
                      (when (member 2 choices) (setf labels (send self :active-labels)))
                      (when (> (length file) 3)
                            (cond
                              ((string= ".lsp" file :start2 (- (length file) 4))
                               (setf file (string-right-trim ".lsp" file)))
                              ((string= ".txt" file :start2 (- (length file) 4))
                               (setf file (string-right-trim ".txt" file)))))
                      (setf file (strcat (string file) ".txt"))
                      (format t "; ExPort: ~a~%> " file)
                      
                      (let ((f (open file :direction :output))
                            (oldbreak *breakenable*)
                            (asym?))

                        (setq *breakenable* nil)
                        
                        (unwind-protect
                         
                         (when (member 2 choices)
                               (setf varnames 
                                     (combine (strcat "ViSta:" (send self :name))
                                              varnames))
                               (if mattype 
                                   (setf asym? (not (not 
                                           (member "asymmetric" 
                                            (map-elements #'string-downcase (send $ :shapes))
                                            :test #'equal)))))
                               (setf typenames 
                                     (if mattype
                                         (combine (if asym? "Asymmetric" "Symmetric")
                                                  typenames)
                                         (combine (send self :datatype) typenames))))

                         (when (member 0 choices)                         
                               (mapcar #'(lambda (var) (format f "~s " var)) varnames)
                               (terpri f))
                         (when (member 1 choices)
                               (mapcar #'(lambda (var) (format f "~s " var)) typenames)
                               (terpri f))
                         (cond
                           (mattype
                            (let ((matrix-names (send self :active-matrices '(all)))
                                  (shapes (send self :active-shapes '(all)))
                                  (nmat (send self :nmat))
                                  (nvar (send self :nvar)))
                              (dotimes (j nmat)
                                       (dotimes (i nvar)
                                                (when (member 2 choices)
                                                      (format f "~s "
                                                              (strcat
                                                               (select matrix-names j)
                                                               ":"
                                                               (string-capitalize
                                                                (select shapes j))
                                                               ":"
                                                               (select labels i))))
                                                (mapcar #'(lambda (val) 
                                                            (format f "~a " val))
                                                        (coerce (select datarow i) 'list))
                                                (terpri f)))))
                           (t
                            (dotimes (i nobs) 
                                     (when (member 2 choices)
                                           (format f "~s " (select labels i)))
                                     (mapcar #'(lambda (val) 
                                                 (format f "~a " val))
                                             (coerce (select datarow i) 'list))
                                     (terpri f))
                            ))
                         (setq *breakenable* oldbreak)
                         (close f)
                         f)))))
    t))



(defun import-data (&optional file (variables nil set) 
                              (types nil typeset) 
                              (labels nil labset)
                              (datatype "multivariate" datatypeset))
"Args: (&optional FILE)
Imports a flat, rectangular ascii files as multivariate or matrix data. Data are read from FILE. A dialog box is presented if FILE is NIL. A file whose contents are a rectangular dataset is a multiple record (line) file, where each record has the same number of data elements, a data element being a number, a symbol or a string. Symbols are converted to upper case, strings retain original case. Missing data elements are represented by the symbol nil or NIL. The string "nil" (regardless of case) is interpreted as a string, not as a missing data-element. The first record usually specifies the data and variable names, and the second record usually specifies the data and variable types. See the Help file for more information."

  (let* ((data) (name) (vista?) (datatype) (matrix?) 
         (nrows) (nvar) (nobs) (nmat) (ncols) (flag)
         (varnames) (vartypes) (obslabel) (matnames) 
         (matshape) (datasheet?) (d) (datum) (types))
    (ignore-errors (setf data (read-data-columns)))
    (when data (format t "; ImPort: ~a~%> " *read-data-file-name*))
    (unless data (fatal-message "File does not contain importable data"))
    (setf name (select (select data 0) 0))
    (setf vista? 
          (when (> (length name) 4)
                (equal (string-downcase (select name (iseq 5))) "vista")))
    (when (and vista? (> (length name) 6))
          (setf name (select name (iseq 6 (1- (length name))))))
    (setf datatype (string-downcase (select (select data 0) 1)))
    (setf freq?    (or (equal datatype "frequency")
                       (equal datatype "freqclass")
                       (equal datatype "crosstabs")))
    (setf matrix? (or (equal datatype "symmetric")
                      (equal datatype "asymmetric")))
    (setf type? (cond (freq? 0) (matrix? 1) (t 2)))  
(break)
    (cond
      ((or vista?
           (member (string-downcase datatype)
                   '("category" "univariate" "bivariate" 
                                "multivariate" "classification" "frequency" 
                                "freqclass" "crosstabs" "general" "missing" 
                                "symmetric" "asymmetric")))
       (import-vista-exported-data data name datatype matrix? freq? type?)
       )

      (t
       (mapcar #'(lambda (val1 val2)
                   (unless flag
                           (unless (stringp val1) (setf flag t))
                           (unless (stringp val2) (setf flag t))))
               (select data 0) (select data 1))
       (if flag 
           (import-plain-data data)
           (import-unknown-exporter-data data name datatype matrix? freq? type? flag))))
    ))

(defun import-plain-data (data)
  (let* ((missing?)
         (types)
         (nvar (length data))
         (nobs (length (first data)))
         (dob)
         )
    (setf types
          (mapcar #'(lambda (var)
                      (unless missing? (setf missing? (position 'nil var)))
                      (if (some 'stringp var) "Category" "Numeric"))
                  data))
    (setf dob
          (data "PlainData"
                :title "PlainData"
                :types types
                :freq nil
                :missing-values missing?
                :labels nil
                :variables (mapcar #'(lambda (x) (format nil "Var~a" x)) (iseq nvar))
                :data (combine (transpose (matrix (list nvar nobs) (combine data))))
                ))
    (send dob :info)
    ))

(defun import-unknown-exporter-data (data name datatype matrix? freq? freq-matrix-type? no-names-or-types)
  (let* ((options "OPTIONS")
         (datatype (send text-item-proto :new "THE DATA ARE:"))
         (filelines (send text-item-proto :new "FILE FORMAT"))
         (data-title-text (send text-item-proto :new "DATA TITLE:"))
         (data-title (send edit-text-item-proto :new "Data Object Name"))
         (first-line (send toggle-item-proto :new 
                           (format nil "FIRST ROW: Variable Names?")
                                :value t))
         (second-line (send toggle-item-proto :new
                            (format nil "SECOND ROW: Variable Types?")
                            :value t))
         (first-column (send toggle-item-proto :new
                             (format nil "FIRST COLUMN: Observation Labels?")
                             :value t))
         (type? (send choice-item-proto :new (list "Frequencies" "Distances" "Neither")
                      :value freq-matrix-type?))
         (spacer1 (send text-item-proto :new "   "))
         (spacer2 (send text-item-proto :new "   "))
         (datasheet? nil)
         (missing? (send toggle-item-proto :new "Check for Mising Values?"))
         (strings? (send toggle-item-proto :new (format nil "All Categories are Strings?")))
         (ok (send modal-button-proto :new "Import"
                   :action #'(lambda () 
                               (list
                                (send data-title :text)
                                (send first-line :value)
                                (send second-line :value)
                                (send first-column :value)
                                nil
                                (send missing? :value)
                                (send type? :value)))))
         (cancel (send modal-button-proto :new "Cancel"))
         (help   (send button-item-proto :new "Help" 
                       :action #'(lambda () (get-menu-item-help IMPORT-DATA-FILE-MENU-ITEM))))
         (dialog (send modal-dialog-proto :new
                       (list (list data-title-text data-title)
                             (list
                              (list filelines (list spacer2 
                                                    (list first-line 
                                                          second-line 
                                                          first-column))
                                    datatype (list spacer1 type?))
                                   (list options
                                         missing? strings?
                                         ok cancel help)))
                       :title "Import Data"))
         (result nil)
         )
    (setf result (send dialog :modal-dialog))
    (when result
          (setf data-object-title (first result))
          (setf variables (second result))
          (setf types (third result))
          (setf labels  (fourth result))
          (setf datasheet? (fifth result))
          (setf missing? (sixth result))
          (setf freqdata? (seventh result))
          (setf nvar (length data))
          (setf nobs (length (first data)))
          )
    (when data 
          (when variables
                (setf variables (repeat nil nvar))
                (dotimes (i nvar)
                         (setf (select variables i) (select (select data i) 0)))
                (setf data (mapcar #'(lambda (var) (rest var)) data))
                (setf nobs (1- nobs))
                (when types
                      (setf types nil)
                      (dotimes (i nvar)
                               (setf types (append types (list (select (select data i) 0)))))
                      (setf data (mapcar #'(lambda (var) (rest var)) data))
                      (setf nobs (1- nobs)))
                (when (or labels 
                          (equal (string-downcase (string (first variables))) "label")
                          (equal (string-downcase (string (first variables))) "labels"))
                      (setf variables (rest variables)))
                )
          (when labels
                (setf labels (first data))
                (setf data (rest data))
                (setf nvar (1- nvar))
                (when types (setf types (rest types)))
                )
          
          (when (not types)
                (setf types (repeat "Numeric" nvar))
                (dotimes (i nvar)
                         (dotimes (j nobs)
                                  (setf datum (select (select data i) j))
                                  (when (or (symbolp datum) (stringp datum))
                                        (when (not (equal datum nil))
                                              (setf (select (select data i) j)
                                                    (string (select (select data i) j)))
                                              (setf (select types i) "Category"))))))
          (setf data 
                (data data-object-title 
                      :title data-object-title 
                           :types types
                      :freq freqdata?
                      :missing-values missing?
                      :labels (if labels (mapcar #'string labels) nil)
                      :variables (if variables (mapcar #'string variables)
                                     (mapcar #'(lambda (x) 
                                                 (format nil "Var~a" x)) (iseq nvar)))
                      :data (combine (transpose (matrix (list nvar nobs) 
                                                        (combine data))))))
          (setcd data)
          (when datasheet? (edit-data data)))
    data))
      
(defun extract-mat-names (labels)
  (mapcar #'(lambda (label)
              (setf pstn (position #\: label :test #'equal))
              (select label (iseq pstn)))
          labels))

(defun extract-mat-shapes (labels)
  (mapcar #'(lambda (label)
              (setf pstn (position #\: label :test #'equal))
              (select label (iseq (+ 1 pstn) (+ 4 pstn))))
          labels))

(defun import-vista-exported-data (data name datatype matrix? freq? type?)
  (let* ((nrows) (nvar) (nobs) (nmat) (ncols) 
         (varnames) (vartypes) (obslabel) (matnames) (matshape))
    (setf data (transpose (matrix (list (length data) (length (first data)))
                                  (combine data))))
    (setf ncols (select (array-dimensions data) 1))
    (setf nrows (select (array-dimensions data) 0))
    (setf varnames (combine (select data 0 (iseq 1 (1- ncols)))))
    (setf vartypes (combine (select data 1 (iseq 1 (1- ncols)))))
    (setf obslabel (combine (select data (iseq 2 (1- nrows)) 0)))
    (setf data     (combine (select data (iseq 2 (1- nrows)) 
                                    (iseq 1 (1- ncols)))))
    (cond
      (matrix?
       (setf nvar     (length varnames))
       (setf nobs     (length obslabel))
       (setf nmat (/ nobs nvar))
       (unless (integerp nmat) (fatal-message "File does not contain importable data"))
       (setf matnames (remove-duplicates (extract-mat-names obslabel) :test #'equal))
       (setf matshape (select (extract-mat-shapes obslabel) (* nvar (iseq nmat))))
       (data name
             :variables varnames
             :types     vartypes
             :labels    obslabel
             :datatype  datatype
             :matrices  matnames
             :shapes    matshape
             :data      data))
      (t
       (data name
             :variables varnames
             :types     vartypes
             :labels    obslabel
             :datatype  datatype
             :freq      freq?
             :data      data)
       ))))